import pickle

import requests
from pyquery import PyQuery as pq


def get_html(url, code="utf-8"):
    try:
        headers = {"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
                                 "(KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36"}
        r = requests.get(url, headers=headers)
        r.raise_for_status()
        r.encoding = code
        return r.text
    except:
        return ""


def get_dir(html):
    doc = pq(html)
    dirs = doc("div #chapter-list-1 a")
    print((type(dirs.items())))
    lst = []
    for dir in dirs.items():
        lst.append(dir.attr('href'))
    with open("dir_lst.pickle", "wb") as f:
        pickle.dump(lst, f)
        f.close()


def main():
    url = "https://www.manhuadui.com/manhua/chuanwu/"
    html = get_html(url)
    get_dir(html)


if __name__ == "__main__":
    main()
